import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
import java.util.UUID;
import org.ansj.app.keyword.KeyWordComputer;
import org.ansj.app.keyword.Keyword;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;

import mblog.web.qiniu.UploadImg;
import mtons.modules.utils.PreviewTextUtils;

public class JianShu {
	static String SITE = "http://www.jianshu.com";// 月热门";
	static String NOW_PAGE_URL_FILE = "nowPageURL.txt";// 月热门";
	static Map<String, String> cookies = null;
	static String userAgent = null;
	static String MBLOG_JSESSIONID = null;

	{
		/************************* CONFIG *********************/
		MBLOG_JSESSIONID = "61973D76BCF24C681EBBAD3261918BE2";// MBLOG SESSION
		cookies = new HashMap<>();
		userAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36";
		/************************* CONFIG *********************/
	}

	String TYSSPIDERLOGDIR = "D:/luoshSpiderLog/"; // "D:/tysSpiderLog/";
	String NEW_PIC_DIR = "D:/luoshSpiderLog/jianshu_pic/";
	String QINIU_DOMAIN="http://luosh.qiniudn.com/";

	@Test
	public void processFace() {
		String url = "";
		//获取上一次读取页面，继续开始
		String nowPageUrl = getNowPageUrl();
		if(StringUtils.isBlank(nowPageUrl)){
			url = SITE+ "/trending/now";
		}else{
			url = nowPageUrl;
		}
		String tid = getArcticleId(url);
		if ("-1".equals(tid)) {
			processMutil(url);
		} else {
			processOne(url);
		}
		//获取下一页内容的请求URL
		String loadNextPageUrl = getLoadNextPageUrl(url);
		loadNextPageUrl.substring(loadNextPageUrl.indexOf("note_ids", 20));
		//记录当前读取的页数URL
		nowPageUrl(loadNextPageUrl);
		processNextPage(loadNextPageUrl);
		System.out.println("over------------------------------------");
	}

	// 处理下一页
	void processNextPage(String loadNextPageUrl) {
		int pageNo = 1;
		String url = loadNextPageUrl;
		String tid = getArcticleId(url);
		if ("-1".equals(tid)) {
			processMutil(url);
		} else {
			processOne(url);
		}
		System.out.println("处理第" + pageNo++ + "页开始！");
		processFace();
	}

	String getLoadNextPageUrl(String url) {
		String nextPageUrl = SITE;
		try {
			Document doc = getDoc(url);
			nextPageUrl += doc.body().select("button.ladda-button ").get(0).attr("data-url");
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return nextPageUrl;
	}

	void processMutil(String url) {
		try {
			Document doc = getDoc(url);
			// System.out.println(doc.body());
			Elements links = doc.body().select("h4.title");
			for (Element link : links) {
				link.select("em").remove();
				Elements alinks = link.select("a");
				// System.out.println(alinks.get(0));
				String alink = alinks.get(0).absUrl("href");
				System.out.println("开始抓取：" + alink);
				processOne(alink);
			}

		} catch (Exception e) {
		}

	}

	void processOne(String url) {
		try {
			// http://www.jianshu.com/p/423cbbe0bf80
			String[] splitStr = url.split("/");
			int len = splitStr.length;
			String tid = splitStr[len - 1];
			try {
				Thread.sleep((long) new Random().nextInt(5000));
			} catch (Exception e) {
			}
			if (!isNewArticle(tid)) {
				System.out.println("文章已抓取过：" + url);
				return;
			}

			Document doc = getDoc(url);
			doc.setBaseUri(url);//
			Map<String, String> article = parseDoc(doc);
			if (article == null) {
				return;
			}
			postRsltMap(article);
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	/**
	 * 发送到mblog
	 * 
	 * @param artcleMap
	 * @throws IOException
	 */
	void postRsltMap(Map<String, String> artcleMap) {
		if (null == (artcleMap.get("content"))) {
			artcleMap.get("errorUrl");
			return;
		}

		MblogPost.post(artcleMap);
	}

	/**
	 * 获取DOCUMENT对象
	 * 
	 * @param url
	 * @return
	 * @throws IOException
	 */
	Document getDoc(String url) throws IOException {
		Response resp = Jsoup.connect(url).userAgent(userAgent).cookies(cookies).execute();
		cookies = resp.cookies();
		// System.out.println("访问："+url);
		System.out.println("返回：" + cookies);
		Document doc = resp.parse();
		return doc;
	}

	/***
	 * 解析DOCUMENT
	 * 
	 * @param doc
	 * @return
	 * @throws IOException
	 */
	Map<String, String> parseDoc(Document doc) {
		Map<String, String> artcleMap = new HashMap<>();
		try {
			artcleMap.put("srcUrl", "" + doc.baseUri());
			String title = doc.title().replace("- 简书", "").replace("简书", "落事");
			artcleMap.put("title", title);
			Element bodyObj = doc.body();
			Elements content = bodyObj.select("div.show-content");
			artcleMap.put("summary", PreviewTextUtils.getText(content.text(), 126));
			// 替换所有超链接为空事件
			content.select("a[href]").attr("href", "javascript:void()");
			// 替换图片
			Elements imgObjs = content.select("img[src]");
			for (Element img : imgObjs) {
				img.attr("src", getNewImgUrl(img.attr("abs:src")));
			}
			artcleMap.put("content", content.toString());
			artcleMap.put("tags", getTag(title, content.text()));
			artcleMap.put("created", new Date() + "");
			saveHtmFile(content.toString());
		} catch (Exception e) {
			artcleMap.put("content", "管理员" + doc.baseUri());
			artcleMap.put("tags", "管理员");
			return artcleMap;
		}

		return artcleMap;
	}

	void saveHtmFile(String text) throws IOException {
		String fileName = UUID.randomUUID() + "12.html";
		FileWriter fw = new FileWriter(new File(NEW_PIC_DIR + fileName));
		fw.write(text);
		fw.close();
	}

	String getNewImgUrl(String imgUrlStr) throws IOException {
		String luoshPicPath = "";
		System.out.println("保存图片：" + imgUrlStr);
		String fileName = UUID.randomUUID() + ".png";// 暂时写死PNG
		// jpg, jpeg, gif, png
		if (imgUrlStr.endsWith(".jpg")) {
			fileName = fileName.substring(0, fileName.length() - 4) + ".jpg";
		} else if (imgUrlStr.endsWith(".jpeg")) {
			fileName = fileName.substring(0, fileName.length() - 4) + ".jpeg";
		} else if (imgUrlStr.endsWith(".gif")) {
			fileName = fileName.substring(0, fileName.length() - 4) + ".gif";
		}

		/*URL imgUrl = new URL(imgUrlStr);
		URLConnection uc = imgUrl.openConnection();
		InputStream is = uc.getInputStream();
		OutputStream os = new FileOutputStream(new File(NEW_PIC_DIR + fileName));
		IOUtils.copy(is, os);*/
		try {
			new UploadImg().upload(imgUrlStr, fileName);
			luoshPicPath =  QINIU_DOMAIN + fileName;
		} catch (Exception e) {
			luoshPicPath =  "http://luosh.qiniudn.com/1083963-d4743637b8fbdb6a.jpg";
		}
		return luoshPicPath;
	}

	/**
	 * 检查该文章是否已读取
	 * 
	 * @param tid
	 * @return
	 */
	Boolean isNewArticle(String tid) {
		String fileName = "credit51.txt";
		try {
			File f = new File(TYSSPIDERLOGDIR + fileName);
			if (FileUtils.readFileToString(f).contains(tid)) {

			} else {
				FileUtils.writeStringToFile(f, tid + ";", true);
				return true;
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
		return false;
	}

	/**
	 * 记录需要手动的文章URL
	 * 
	 * @param url
	 * @return
	 */
	Boolean manmalArticle(String url) {
		String fileName = "credit51_err.txt";

		try {
			File f = new File(TYSSPIDERLOGDIR + fileName);
			FileUtils.forceMkdir(f);
			FileUtils.writeStringToFile(f, url + "\n", true);
		} catch (IOException e) {
			e.printStackTrace();
		}
		return false;
	}
	/**
	 * 记录当前读取的页面URL
	 * 
	 * @param url
	 * @returnd
	 */
	Boolean nowPageUrl(String url) {
		String urla = "";
		String urlb = getNowPageUrl();
		if(!StringUtils.isBlank(urlb)){
			urlb=urlb.substring(urlb.lastIndexOf("?")+1, urlb.length());
			urlb = urlb.substring(0, urlb.lastIndexOf("page"));
			url=url.replaceAll(urlb, "");
			urla = url;
		}else{
			urla = url;
		}
		try {
			File f = new File(TYSSPIDERLOGDIR + NOW_PAGE_URL_FILE);
			if(f.exists()){  
				f.delete();  
			}
			f.createNewFile();
			//FileUtils.forceMkdir(f);
			FileUtils.writeStringToFile(f, urla + "\n", true);
		} catch (IOException e) {
			e.printStackTrace();
		}
		return false;
	}

	/**
	 * 获取当前页URL
	 * 
	 * @param tid
	 * @return
	 */
	String getNowPageUrl() {
		String NOW_PAGE_URL="";
		try {
			File f = new File(TYSSPIDERLOGDIR + NOW_PAGE_URL_FILE);
			NOW_PAGE_URL =  FileUtils.readFileToString(f);
		} catch (IOException e) {
			e.printStackTrace();
		}
		return NOW_PAGE_URL;
	}
	String getArcticleId(String url) {
		// http://bbs.51credit.com/forum.php?mod=viewthread&tid=1037839&extra=page%3D1%26orderby%3Dheats
		String[] pArr = url.split("&");
		for (String p : pArr) {
			if (p.startsWith("tid=")) {
				return p.replaceAll("tid=", "");
			}
		}
		return "-1";
	}
	//获取标签
	String getTag(String title, String article) {
		KeyWordComputer kwc = new KeyWordComputer(5);
		Collection<Keyword> result = kwc.computeArticleTfidf(title, article);
		String str = result.toString().replaceAll("\\d+", "").replaceAll("/.", "");
		String[] tagArr = str.substring(1, str.length() - 1).split(", ");
		String tmpTag = "";
		for (String t : tagArr) {
			tmpTag += t + ",";
		}
		tmpTag = tmpTag.substring(0, tmpTag.length() - 1);
		System.out.println("标签：" + tmpTag);
		return tmpTag;
	}

}
